# ------------------------------------------------------------------------------
#    Collection of utility functions to help interacting with the 'adb'
#    and 'android' command line tools.
# ------------------------------------------------------------------------------

# ------------------------------------------------------------------------------
#    Python standard library modules.
# ------------------------------------------------------------------------------
import os
import sys
import subprocess
import re
from pprint import pprint
import ConfigParser as configparser
import tempfile
import shutil
from glob import glob
import time
import threading
import datetime
# ------------------------------------------------------------------------------

# ------------------------------------------------------------------------------
#    My modules.
# ------------------------------------------------------------------------------
import utils
from ResultsManager import ResultsManager
from AVD import AVD
from SCM import SCMHandler, SCMRevision
# ------------------------------------------------------------------------------

# ------------------------------------------------------------------------------
#    Set up the logger.  Set the logger object to something you own if you want
#    to change it.
# ------------------------------------------------------------------------------
import logging
global logger
# create logger
logger = logging.getLogger("")
logger.setLevel(logging.DEBUG)
# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
# add formatter to ch
ch.setFormatter(formatter)
# add ch to logger
logger.addHandler(ch)
# ------------------------------------------------------------------------------

CMD_ANDROID = "android"
CMD_ADB = "adb"
CMD_ANT = "ant"
CMD_EMULATOR = "emulator"

CMD_CTAGS = "ctags"

# ------------------------------------------------------------------------------
#    Regular expressions.
# ------------------------------------------------------------------------------
RE_GET_TARGETS_START    = re.compile("Available Android targets:")
RE_GET_TARGETS_ID       = re.compile("id:[\s]+[/d]+ or \"(.*)\"")
RE_GET_TARGETS_NAME     = re.compile("Name:[\s]+(.*)")
# ------------------------------------------------------------------------------


class EmulatorInstance(object):
    """ A wrapper around a particular instance of a "emulator -avd <avd_name>"
    emulator running.  Each instance of this object will execute two processes:
    one to hold the emulator, another to continually collect the logcat logs.
    """
    
    class LogcatThread(threading.Thread):
        """ Thread to keep track of the logcat output from an emulator.
        """
        
        def __init__(self, emulator_name, tempdir):
            threading.Thread.__init__(self)
            self.log = logging.getLogger("LogcatThread::%s" % (emulator_name, ))
            self.emulator_name = emulator_name
            self.is_running = threading.Event()
            self.is_stopped = threading.Event()
            self.output = []
            self.output_lock = threading.Lock()
            self.tempdir = tempdir
            assert(os.path.isdir(self.tempdir))
            self.logcat_filename = os.path.join(self.tempdir, "%s_logcat.txt" % (emulator_name, ))
            self.f = open(self.logcat_filename, "w")
            
        def run(self):
            """ Tail the logcat output of the emulator instance.
            (e.g. adb -s emulator-5554 logcat -v time)
            """
            cmd = "%s -s %s logcat -v time" % (CMD_ADB, self.emulator_name)
            self.log.debug("Executing \"%s\"." % (cmd, ))
            self.proc = subprocess.Popen("%s" % (cmd, ), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            self.is_running.set()
            try:
                while self.is_running.is_set():                    
                    o = self.proc.stdout.readline()
                    if o == '' and self.proc.poll() != None:
                        # emulator has unexpectedly exited or never started to
                        # begin with
                        self.log.debug("Device unexpectedly no longer running.")
                        self.is_running.clear()
                    elif o != '':
                        self.output_lock.acquire()                                                    
                        self.output.append(o.strip())
                        self.f.write(o.strip() + "\n")
                        self.output_lock.release()
                            
            finally:                
                # at this point we've been told to stop, so clean up the
                # process.
                if self.proc.poll() == None:
                    self.proc.terminate()
                    time.sleep(3)
                    if self.proc.poll() == None:
                        self.proc.kill()
                        time.sleep(3)   
                self.is_stopped.set()
                self.f.close()
                
        def get_output(self):
            self.output_lock.acquire()
            output = self.output[:]
            self.output_lock.release()
            return output        
                
        def stop(self):
            self.is_running.clear()
            self.is_stopped.wait(10.0)
            
        def wait_for(self, regexp, timeout=None):
            """ Wait for a particular regular expression to match in the
            logcat output.  If timeout is None will wait forever until a match
            is found.  Else, after timeout seconds return False. 
            """
            regexp_compiled = re.compile(regexp)
            last_line_checked = 0
            time_start = time.time()
            while 1:
                time.sleep(0.1)                
                self.output_lock.acquire()
                try:
                    for i in xrange(last_line_checked, len(self.output)):
                        if regexp_compiled.match(self.output[i]) is not None:
                            return True
                finally:
                    last_line_checked = len(self.output)
                    self.output_lock.release()
                if not self.is_running.is_set():
                    return False
                if timeout and (time.time() - time_start > timeout):
                    return False                                                             
    
    def __init__(self, avd_name, tempdir):
        self.logger = logging.getLogger("EmulatorInstance::%s" % (avd_name, ))
        self.execute_lock = threading.Lock()
        
        # ----------------------------------------------------------------------
        #    Launch the AVD.
        # ----------------------------------------------------------------------
        self.avd_name = avd_name
        devices_old = set(get_running_devices())        
        cmd = "%s -avd %s -wipe-data" % (CMD_EMULATOR, avd_name)
        self.logger.debug("Execute \"%s\"." % (cmd, ))
        self.proc = subprocess.Popen("%s" % (cmd, ))        
        time.sleep(5)
        # ----------------------------------------------------------------------
        
        # ----------------------------------------------------------------------
        #    Use the difference in the output of 'adb devices' to determine
        #    which device is running our AVD, then start tracking its logcat
        #    output.
        # ----------------------------------------------------------------------
        devices_new = set(get_running_devices())
        assert(devices_old.issubset(devices_new))
        assert(len(devices_new - devices_old) is 1)
        self.my_device = (list(devices_new - devices_old))[0]
        self.logcat = self.LogcatThread(self.my_device, tempdir)
        self.logcat.start()
        
        # TODO how do I determine when the emulator has finished loading?
        # this hack depends on a debug line from the PowerManagerService,
        # which is risky.
        self.logger.debug("Waiting for emulator to come up...")
        self.logcat.wait_for(".*SurfaceFlinger.*Boot is finished", timeout=120)        
        self.logger.debug("Emulator is up.")
        # ----------------------------------------------------------------------        
        
    def execute_command(self, cmd):
        """ Wrapper around 'adb -s <emulator_name> <command>'
        """
        
        self.execute_lock.acquire()
        cmd = "%s -s %s %s" % (CMD_ADB, self.my_device, cmd)
        self.logger.debug("Executing \"%s\"" % (cmd, ))
        result = utils.execute_command(cmd)
        self.execute_lock.release()
        return result        
      
    def close(self):
        """ Clean up any running threads and other resources.
        """       
        if self.proc.poll() == None:
            # still running.  kill it.
            self.logger.debug("Still running.  Send SIGTERM.")
            self.proc.terminate()
            time.sleep(3)
            if self.proc.poll() == None:
                self.logger.debug("Still running after SIGTERM.  Send SIGKILL.")
                self.proc.kill()
        self.logcat.stop()       


class ExecutionManager(object):
    """ Deals with executing tests.
    """
    
    RE_ANT_BUILD_FAILED = re.compile(".*BUILD FAILED", re.DOTALL)
    RE_PACKAGE_MANAGER_NOT_LOADED = re.compile(".*(Could not access the Package Manager|DDM dispatch reg wait timeout)", re.DOTALL)
    RE_ACTIVITY_MANAGER_NOT_LOADED = re.compile(".*(Can't connect to activity manager)", re.DOTALL)
    
    def __init__(self, scm_handler, tempdir):
        self.scm_handler = scm_handler
        self.tempdir = tempdir
        assert(os.path.isdir(self.tempdir))
        
    def execute_all_tests_on_target(self, target, revision_index):    
        """ Execuate all available tests on one particular target (i.e. an
        Android Virtual Device) and on one particular revision.  The revision
        is specified as an integer index into the list of revisions returned
        by SCMHandler::get_revisions() (0 is the newest, -1 is the oldest).
        """    
        logger.debug("ENTRY ExecutionManager::execute_all_tests_on_target: target=%s" % (target, ))
        
        # ----------------------------------------------------------------------
        #    Prepare the temporary checked-out copy of the source code we want
        #    to execute tests against.
        # ----------------------------------------------------------------------
        revisions = scm_handler.get_revisions()
        assert(revision_index >= -1 and revision_index < len(revisions))        
        scm_handler.clone_to_dir(revisions[revision_index], self.tempdir)
        os.chdir(self.tempdir)
        results = utils.execute_command([CMD_ANDROID, "update", "project", "--path", "."])
        logger.debug("Executed \"android update project --path .\" in directory \"%s\":\n%s", os.getcwd(), results)
        results = utils.execute_command([CMD_ANT, "debug"])
        logger.debug("Executed \"ant debug\" in directory \"%s\":\n%s" % (os.getcwd(), results))
        if self.RE_ANT_BUILD_FAILED.match(results) is not None:
            # build failed, so bail out now.
            logger.error("Build failed. Check 'ant debug' output.")
            return False
        # ----------------------------------------------------------------------
        
        self.clone_scm_handler = SCMHandler(self.tempdir, scm_handler.method)        
        binary_file = self.clone_scm_handler.get_binary_file() 
        logger.debug("Using binary file \"%s\"." % (binary_file, ))        
        
        # ----------------------------------------------------------------------
        #    Start the emulator, tail its logcat output, and execute the tests.
        # ----------------------------------------------------------------------
        self.emulator = EmulatorInstance(target, self.tempdir)        
        try:                    
            logger.debug("Package name: \"%s\"" % (scm_handler.package_name, ))
            
            # ------------------------------------------------------------------
            #    Uninstall then install the binary.  Note that, as per the
            #    'adb' manual, there is no standard way for determining when
            #    the emulator has finished booting.  Prepending
            #    "wait-for-device" is not a substitute, as this verifies a
            #    stage of loading before the booting is finished.  Hence, we're
            #    going to repeatedly attempt to uninstall the binary until
            #    the package manager comes up.
            # ------------------------------------------------------------------
            logger.debug("Uninstalling old binary:")
            while 1:                                                
                result = self.emulator.execute_command("uninstall %s" % (scm_handler.package_name, ))
                logger.debug(result)                
                if self.RE_PACKAGE_MANAGER_NOT_LOADED.match(result) is None:                    
                    break
                logger.debug("Package manager not loaded.  Try again.")
                time.sleep(5)
                            
            logger.debug("Installing new binary:")
            result = self.emulator.execute_command("install %s" % (binary_file, ))
            logger.debug(result)        
            # ------------------------------------------------------------------
            
            logger.debug("Executing tests...")
            while 1:                
                result = self.emulator.execute_command("shell am instrument -w %s/android.test.InstrumentationTestRunner" % (scm_handler.package_name, ))
                if self.RE_ACTIVITY_MANAGER_NOT_LOADED.match(result) is None:
                    logger.debug(result)                    
                    break
                time.sleep(5)
            logger.debug("Finished executing tests.")
        finally:                           
            self.emulator.close()
            
        logger.debug("EXIT ExecutionManager::execute_all_tests_on_target")
        return True
    
    def get_logcat_filename(self):
        """ Return the location of the logcat output file.
        """
        assert(os.path.isfile(self.emulator.logcat.logcat_filename))
        return self.emulator.logcat.logcat_filename
    
    def get_test_names(self):
        """ Return the expected test names for this execution run. The
        ExecutionManager will create a SCMHandler for the temporary directory
        that holds the current revision being tested and use
        SCMHandler::get_test_names().
        """
        return self.clone_scm_handler.get_test_names()

def verify_environment():
    cmds_to_check = [ ("Is 'android' present", [CMD_ANDROID, "--help"], "Android Virtual Device"),
                      ("Is 'adb' present", [CMD_ADB, "--help"], "Android Debug Bridge"),
                      ("Is 'ant' present", [CMD_ANT, "-version"], "Apache Ant version"),
                      ("Is 'emulator' present", [CMD_EMULATOR, "-version"], "Android emulator version"),
                      ("Is 'hg' present", [CMD_HG, "--version"], "Mercurial Distributed SCM"),
                      ("Is 'ctags' present", [CMD_CTAGS, "--version"], "Exuberant Ctags")]
    cmds_results = []
    for (msg, cmd, verification_str) in cmds_to_check:    
        result = utils.execute_command(cmd)
        if verification_str in result:
            print "%s...yes." % (msg, )
            cmds_results.append(True)
        else:
            print "%s...no." % (msg, )
            cmds_results.append(False)
    
    global IS_ENVIRONMENT_VERIFIED        
    IS_ENVIRONMENT_VERIFIED = False not in cmds_results
    return IS_ENVIRONMENT_VERIFIED
    
RE_GET_DEVICES_START    = re.compile("Available Android Virtual Devices")
RE_GET_DEVICES_NAME     = re.compile("\s*Name:\s+(.*)")
RE_GET_DEVICES_PATH     = re.compile("\s*Path:\s+(.*)")
RE_GET_DEVICES_TARGET   = re.compile("\s*Target:\s+(.*)")
RE_GET_DEVICES_TARGET_GOOGLE   = re.compile("\s*Target:\s+Google APIs")
RE_GET_DEVICES_SKIN     = re.compile("\s*Skin:\s+(.*)")
RE_GET_DEVICES_SDCARD   = re.compile("\s*Sdcard:\s+(.*)")
RE_GET_DEVICES_LINE     = re.compile("^[-]{5,}")    

def get_devices():    
    """
    e.g. 
    I:\eclipse_workspace\sisyphus\src>android list avd
    Available Android Virtual Devices:
        Name: 1.5_HVGA
        Path: C:\Users\ai\.android\avd\1.5_HVGA.avd
      Target: Android 1.5 (API level 3)
        Skin: HVGA    
        
    Remember that "Sdcard:" is optional.
    """
    
    logger = logging.getLogger("get_devices")
    cmd = [CMD_ANDROID, "list", "avd"]
    logger.debug("executing \"%s\"" % (' '.join(cmd)))
    result = utils.execute_command(cmd)
    logger.debug("result: %s" % (result, ))    
    lines = result.splitlines()    
    assert(RE_GET_DEVICES_START.match(lines[0]) is not None)
    (name, path, target, skin, sdcard) = (None, None, None, None, None)
    targets = []
    for i, line in enumerate(lines[1:]):
        m = RE_GET_DEVICES_NAME.match(line)        
        if m is not None:            
            # ------------------------------------------------------------------
            #    At this stage we may have found a new target in the list.
            # ------------------------------------------------------------------
            if all(elem is not None for elem in (name, path, target, skin)):
                targets.append(AVD(name, path, target, skin, sdcard))
                (name, path, target, skin, sdcard) = (None, None, None, None, None)
            name = m.group(1)
            continue
        m = RE_GET_DEVICES_PATH.match(line)        
        if m is not None:            
            path = m.group(1)
            continue
        m = RE_GET_DEVICES_TARGET.match(line)        
        if m is not None:            
            # this might be an odd "Google APIs" target, in which case the
            # string we want is actually on the next line.  just grab it if so.
            if RE_GET_DEVICES_TARGET_GOOGLE.match(line):
                target = lines[i+1].strip()
            else:            
                target = m.group(1)
            continue
        m = RE_GET_DEVICES_SKIN.match(line)        
        if m is not None:            
            skin = m.group(1)
            continue
        m = RE_GET_DEVICES_SDCARD.match(line)        
        if m is not None:            
            sdcard = m.group(1)
            continue        
        m = RE_GET_DEVICES_LINE.match(line)
        if m is not None:
            targets.append(AVD(name, path, target, skin, sdcard))
            (name, path, target, skin, sdcard) = (None, None, None, None, None)            
    targets.append(AVD(name, path, target, skin, sdcard))        
    return targets
    
def get_targets():
    result = utils.execute_command([CMD_ANDROID, "list", "target"])
    
def read_settings(filepath="settings.ini"):
    assert(os.path.isfile(filepath))
    config = configparser.SafeConfigParser()
    config.read(filepath)
    
    sourcecode_path = config.get("Source Code", "fullpath")
    assert(os.path.isdir(sourcecode_path))
    sourcecode_scm_method = config.get("Source Code", "scm_method")
    assert(sourcecode_scm_method in ["hg"])
    global scm_handler
    scm_handler = SCMHandler(sourcecode_path, sourcecode_scm_method)
 
    if config.has_option("Execution", "tempdir"):        
        tempdir = config.get("Execution", "tempdir")
        assert(os.path.isdir(tempdir))
        tempfile.tempdir = tempdir
        
    results_path = config.get("Results", "fullpath")
    assert(os.path.isdir(results_path))
    global results_manager
    results_manager = ResultsManager(scm_handler, results_path) 
    
def execute_all_tests(targets):
    """ Execute all tests on the target names indicated.
    """
    
    logger = logging.getLogger("execute_all_tests")
    
    # can only be none if read_settings() has not been previously called.
    assert(scm_handler is not None)
    assert(results_manager is not None)
    
    # can only be false if verify_environment() has not been previously called.
    assert(IS_ENVIRONMENT_VERIFIED is True)

    # --------------------------------------------------------------------------
    #    Run the tests.
    # --------------------------------------------------------------------------
    revisions = scm_handler.get_revisions()
    logger.debug("targets = %s" % (targets, ))    
    for target in targets:
        tempdir = tempfile.mkdtemp()
        execution_manager = ExecutionManager(scm_handler, tempdir)        
        execution_manager.execute_all_tests_on_target(target, revision_index=0)
        
        # ----------------------------------------------------------------------
        #    Parse the results from the current test run.
        # ----------------------------------------------------------------------
        logcat_file = execution_manager.get_logcat_filename()
        test_runs = results_manager.convert_logcat_file_to_testruns(logcat_file)
        logger.debug("Test runs executed: %s" % (["%s.%s.%s" % (tr.package, tr.parent_class, tr.name) for tr in test_runs], ))
        test_names = execution_manager.get_test_names()
        logger.debug("Test names expected: %s" % (test_names, ))
        # ----------------------------------------------------------------------
        
        # ----------------------------------------------------------------------
        #    Store the results.
        # ----------------------------------------------------------------------
        
        # ----------------------------------------------------------------------    
    
        # ----------------------------------------------------------------------
        #    Clean up the temporary scratch environment.
        # ----------------------------------------------------------------------
        #shutil.rmtree(tempdir)    
        # ----------------------------------------------------------------------
        
    # --------------------------------------------------------------------------
    
    #import pdb
    #pdb.set_trace()        

def get_running_devices():
    """ Use 'adb devices' output to determine which emulators are currently
    running.
    """
    log = logging.getLogger("get_running_devices")
    log.debug("entry")
    RE_ANDROID_EMULATOR_START = re.compile("List of devices attached")
    RE_ANDROID_EMULATOR_LINE = re.compile("(emulator-[\d]+)\s+(device|offline)")
    output = utils.execute_command([CMD_ADB, "devices"])
    devices = []
    for line in output.splitlines():
        m = RE_ANDROID_EMULATOR_LINE.match(line)
        if m is not None:
            devices.append(m.group(1))           
    log.debug("exit.  returning: %s", devices)   
    return devices 
    